{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'albert' already exists and is not an empty directory.\r\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/p208p2002/albert-zh-for-pytorch-transformers.git albert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-03-01 03:22:54--  https://raw.githubusercontent.com/p208p2002/taipei-QA-BERT/master/Taipei_QA_new.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 880722 (860K) [text/plain]\n",
      "Saving to: ‘Taipei_QA_new.txt.1’\n",
      "\n",
      "Taipei_QA_new.txt.1 100%[===================>] 860.08K  2.37MB/s    in 0.4s    \n",
      "\n",
      "2020-03-01 03:22:55 (2.37 MB/s) - ‘Taipei_QA_new.txt.1’ saved [880722/880722]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/p208p2002/taipei-QA-BERT/master/Taipei_QA_new.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir trained_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import TensorDataset\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/usr/lib/python36.zip',\n",
       " '/usr/lib/python3.6',\n",
       " '/usr/lib/python3.6/lib-dynload',\n",
       " '',\n",
       " '/usr/local/lib/python3.6/dist-packages',\n",
       " '/usr/lib/python3/dist-packages',\n",
       " '/usr/local/lib/python3.6/dist-packages/IPython/extensions',\n",
       " '/root/.ipython',\n",
       " '.']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys \n",
    "sys.path.append('.')\n",
    "sys.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from albert.albert_zh import AlbertConfig, AlbertTokenizer, AlbertForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def use_model(model_name, config_file_path, model_file_path, vocab_file_path, num_labels):\n",
    "    # 選擇模型並加載設定\n",
    "    if(model_name == 'bert'):\n",
    "        from transformers import BertConfig, BertForSequenceClassification, BertTokenizer\n",
    "        model_config, model_class, model_tokenizer = (BertConfig, BertForSequenceClassification, BertTokenizer)\n",
    "        config = model_config.from_pretrained(config_file_path,num_labels = num_labels)\n",
    "        model = model_class.from_pretrained(model_file_path, from_tf=bool('.ckpt' in 'bert-base-chinese'), config=config)\n",
    "        tokenizer = model_tokenizer(vocab_file=vocab_file_path)\n",
    "        return model, tokenizer\n",
    "    elif(model_name == 'albert'):\n",
    "        from albert.albert_zh import AlbertConfig, AlbertTokenizer, AlbertForSequenceClassification\n",
    "        model_config, model_class, model_tokenizer = (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer)\n",
    "        config = model_config.from_pretrained(config_file_path,num_labels = num_labels)\n",
    "        model = model_class.from_pretrained(model_file_path, config=config)\n",
    "        tokenizer = model_tokenizer.from_pretrained(vocab_file_path)\n",
    "        return model, tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_accuracy(y_pred, y_target):\n",
    "    # 計算正確率\n",
    "    _, y_pred_indices = y_pred.max(dim=1)\n",
    "    n_correct = torch.eq(y_pred_indices, y_target).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_bert_ids(tokenizer,q_input):\n",
    "    # 將文字輸入轉換成對應的id編號\n",
    "    return tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(q_input)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dataset(input_ids, input_masks, input_segment_ids, answer_lables):\n",
    "    all_input_ids = torch.tensor([input_id for input_id in input_ids], dtype=torch.long)\n",
    "    all_input_masks = torch.tensor([input_mask for input_mask in input_masks], dtype=torch.long)\n",
    "    all_input_segment_ids = torch.tensor([input_segment_id for input_segment_id in input_segment_ids], dtype=torch.long)\n",
    "    all_answer_lables = torch.tensor([answer_lable for answer_lable in answer_lables], dtype=torch.long)    \n",
    "    return TensorDataset(all_input_ids, all_input_masks, all_input_segment_ids, all_answer_lables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_dataset(full_dataset, split_rate=0.8):  \n",
    "    train_size = int(split_rate * len(full_dataset))\n",
    "    test_size = len(full_dataset) - train_size\n",
    "    train_dataset, test_dataset = torch.utils.data.random_split(full_dataset, [train_size, test_size])\n",
    "    return train_dataset,test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataDic(object):\n",
    "    def __init__(self, answers):\n",
    "        self.answers = answers #全部答案(含重複)\n",
    "        self.answers_norepeat = sorted(list(set(answers))) # 不重複\n",
    "        self.answers_types = len(self.answers_norepeat) # 總共多少類\n",
    "        self.ans_list = [] # 用於查找id或是text的list\n",
    "        self._make_dic() # 製作字典\n",
    "    \n",
    "    def _make_dic(self):\n",
    "        for index_a,a in enumerate(self.answers_norepeat):\n",
    "            if a != None:\n",
    "                self.ans_list.append((index_a,a))\n",
    "\n",
    "    def to_id(self,text):\n",
    "        for ans_id,ans_text in self.ans_list:\n",
    "            if text == ans_text:\n",
    "                return ans_id\n",
    "\n",
    "    def to_text(self,id):\n",
    "        for ans_id,ans_text in self.ans_list:\n",
    "            if id == ans_id:\n",
    "                return ans_text\n",
    "\n",
    "    @property\n",
    "    def types(self):\n",
    "        return self.answers_types\n",
    "    \n",
    "    @property\n",
    "    def data(self):\n",
    "        return self.answers\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_data_to_feature(tokenizer, train_data_path):\n",
    "    with open(train_data_path,'r',encoding='utf-8') as f:\n",
    "        data = f.read()\n",
    "    qa_pairs = data.split(\"\\n\")\n",
    "\n",
    "    questions = []\n",
    "    answers = []\n",
    "    for qa_pair in qa_pairs:\n",
    "        qa_pair = qa_pair.split()\n",
    "        try:\n",
    "            a,q = qa_pair\n",
    "            questions.append(q)\n",
    "            answers.append(a)\n",
    "        except:\n",
    "            continue\n",
    "    \n",
    "    assert len(answers) == len(questions)\n",
    "    \n",
    "    ans_dic = DataDic(answers)\n",
    "    question_dic = DataDic(questions)\n",
    "\n",
    "    q_tokens = []\n",
    "    max_seq_len = 0\n",
    "    for q in question_dic.data:\n",
    "        bert_ids = to_bert_ids(tokenizer,q)\n",
    "        if(len(bert_ids)>max_seq_len):\n",
    "            max_seq_len = len(bert_ids)\n",
    "        q_tokens.append(bert_ids)\n",
    "        # print(tokenizer.convert_ids_to_tokens(tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(q)))))\n",
    "    \n",
    "    print(\"最長問句長度:\",max_seq_len)\n",
    "    assert max_seq_len <= 512 # 小於BERT-base長度限制\n",
    "\n",
    "    # 補齊長度\n",
    "    for q in q_tokens:\n",
    "        while len(q)<max_seq_len:\n",
    "            q.append(0)\n",
    "    \n",
    "    a_labels = []\n",
    "    for a in ans_dic.data:\n",
    "        a_labels.append(ans_dic.to_id(a))\n",
    "        # print (ans_dic.to_id(a))\n",
    "    \n",
    "    # BERT input embedding\n",
    "    answer_lables = a_labels\n",
    "    input_ids = q_tokens\n",
    "    input_masks = [[1]*max_seq_len for i in range(len(question_dic))]\n",
    "    input_segment_ids = [[0]*max_seq_len for i in range(len(question_dic))]\n",
    "    assert len(input_ids) == len(question_dic) and len(input_ids) == len(input_masks) and len(input_ids) == len(input_segment_ids)\n",
    "\n",
    "    data_features = {'input_ids':input_ids,\n",
    "                    'input_masks':input_masks,\n",
    "                    'input_segment_ids':input_segment_ids,\n",
    "                    'answer_lables':answer_lables,\n",
    "                    'question_dic':question_dic,\n",
    "                    'answer_dic':ans_dic}\n",
    "    \n",
    "    output = open('trained_model/data_features.pkl', 'wb')\n",
    "    pickle.dump(data_features,output)\n",
    "    return data_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers==2.3.0 in /usr/local/lib/python3.6/dist-packages (2.3.0)\n",
      "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0) (0.1.85)\n",
      "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0) (0.0.38)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0) (1.12.11)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0) (4.43.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0) (1.16.4)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0) (2020.2.20)\n",
      "Requirement already satisfied: requests in /usr/lib/python3/dist-packages (from transformers==2.3.0) (2.18.4)\n",
      "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from sacremoses->transformers==2.3.0) (1.11.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.3.0) (7.0)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.3.0) (0.14.1)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.3.0) (0.3.3)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.3.0) (0.9.5)\n",
      "Requirement already satisfied: botocore<1.16.0,>=1.15.11 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.3.0) (1.15.11)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.11->boto3->transformers==2.3.0) (0.15.2)\n",
      "Requirement already satisfied: urllib3<1.26,>=1.20; python_version != \"3.4\" in /usr/lib/python3/dist-packages (from botocore<1.16.0,>=1.15.11->boto3->transformers==2.3.0) (1.22)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.11->boto3->transformers==2.3.0) (2.8.0)\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0301 03:22:56.929888 140584307713856 file_utils.py:35] PyTorch version 1.3.0+cu100 available.\n",
      "W0301 03:22:57.686077 140584307713856 __init__.py:28] To use data.metrics please install scikit-learn. See https://scikit-learn.org/stable/index.html\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers==2.3.0\n",
    "from torch.utils.data import DataLoader\n",
    "import torch\n",
    "from transformers import AdamW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0301 03:22:57.733967 140584307713856 modeling_albert.py:128] Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n",
      "I0301 03:22:57.735299 140584307713856 configuration_utils.py:145] loading configuration file albert/albert_tiny/config.json\n",
      "I0301 03:22:57.735736 140584307713856 configuration_utils.py:167] Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.0,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"embedding_size\": 128,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.0,\n",
      "  \"hidden_size\": 312,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 1248,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"ln_type\": \"postln\",\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 4,\n",
      "  \"num_labels\": 149,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"share_type\": \"all\",\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "I0301 03:22:57.736531 140584307713856 modeling_utils.py:326] loading weights file albert/albert_tiny/pytorch_model.bin\n",
      "I0301 03:22:57.819349 140584307713856 modeling_utils.py:384] Weights of AlbertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "I0301 03:22:57.819819 140584307713856 modeling_utils.py:387] Weights from pretrained model not used in AlbertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.project_layer.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "I0301 03:22:57.820275 140584307713856 tokenization_utils.py:303] Model name 'albert/albert_tiny/vocab.txt' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc, bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased). Assuming 'albert/albert_tiny/vocab.txt' is a path or url to a directory containing tokenizer files.\n",
      "I0301 03:22:57.821182 140584307713856 tokenization_utils.py:332] Didn't find file albert/albert_tiny/added_tokens.json. We won't load it.\n",
      "I0301 03:22:57.821636 140584307713856 tokenization_utils.py:332] Didn't find file albert/albert_tiny/special_tokens_map.json. We won't load it.\n",
      "I0301 03:22:57.821994 140584307713856 tokenization_utils.py:332] Didn't find file albert/albert_tiny/tokenizer_config.json. We won't load it.\n",
      "I0301 03:22:57.822373 140584307713856 tokenization_utils.py:368] loading file albert/albert_tiny/vocab.txt\n",
      "I0301 03:22:57.822697 140584307713856 tokenization_utils.py:368] loading file None\n",
      "I0301 03:22:57.823016 140584307713856 tokenization_utils.py:368] loading file None\n",
      "I0301 03:22:57.823333 140584307713856 tokenization_utils.py:368] loading file None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using device cuda\n",
      "最長問句長度: 159\n",
      "epoch: 1 batch:   1 train_loss:5.0311 train_acc:6.2500\n",
      "epoch: 1 batch:   2 train_loss:5.0491 train_acc:3.1250\n",
      "epoch: 1 batch:   3 train_loss:5.0474 train_acc:2.0833\n",
      "epoch: 1 batch:   4 train_loss:5.0361 train_acc:1.5625\n",
      "epoch: 1 batch:   5 train_loss:5.0278 train_acc:1.2500\n",
      "epoch: 1 batch:   6 train_loss:5.0090 train_acc:1.0417\n",
      "epoch: 1 batch:   7 train_loss:5.0183 train_acc:0.8929\n",
      "epoch: 1 batch:   8 train_loss:5.0148 train_acc:0.7812\n",
      "epoch: 1 batch:   9 train_loss:5.0055 train_acc:0.6944\n",
      "epoch: 1 batch:  10 train_loss:5.0053 train_acc:0.6250\n",
      "epoch: 1 batch:  11 train_loss:5.0223 train_acc:0.5682\n",
      "epoch: 1 batch:  12 train_loss:5.0231 train_acc:0.5208\n",
      "epoch: 1 batch:  13 train_loss:5.0142 train_acc:0.4808\n",
      "epoch: 1 batch:  14 train_loss:5.0133 train_acc:0.4464\n",
      "epoch: 1 batch:  15 train_loss:5.0241 train_acc:0.4167\n",
      "epoch: 1 batch:  16 train_loss:5.0247 train_acc:0.3906\n",
      "epoch: 1 batch:  17 train_loss:5.0201 train_acc:0.3676\n",
      "epoch: 1 batch:  18 train_loss:5.0149 train_acc:0.3472\n",
      "epoch: 1 batch:  19 train_loss:5.0226 train_acc:0.3289\n",
      "epoch: 1 batch:  20 train_loss:5.0218 train_acc:0.3125\n",
      "epoch: 1 batch:  21 train_loss:5.0240 train_acc:0.2976\n",
      "epoch: 1 batch:  22 train_loss:5.0256 train_acc:0.2841\n",
      "epoch: 1 batch:  23 train_loss:5.0233 train_acc:0.2717\n",
      "epoch: 1 batch:  24 train_loss:5.0172 train_acc:0.2604\n",
      "epoch: 1 batch:  25 train_loss:5.0156 train_acc:0.5000\n",
      "epoch: 1 batch:  26 train_loss:5.0143 train_acc:0.4808\n",
      "epoch: 1 batch:  27 train_loss:5.0147 train_acc:0.4630\n",
      "epoch: 1 batch:  28 train_loss:5.0177 train_acc:0.4464\n",
      "epoch: 1 batch:  29 train_loss:5.0154 train_acc:0.4310\n",
      "epoch: 1 batch:  30 train_loss:5.0130 train_acc:0.4167\n",
      "epoch: 1 batch:  31 train_loss:5.0081 train_acc:0.4032\n",
      "epoch: 1 batch:  32 train_loss:5.0085 train_acc:0.3906\n",
      "epoch: 1 batch:  33 train_loss:5.0065 train_acc:0.3788\n",
      "epoch: 1 batch:  34 train_loss:5.0076 train_acc:0.3676\n",
      "epoch: 1 batch:  35 train_loss:5.0065 train_acc:0.3571\n",
      "epoch: 1 batch:  36 train_loss:5.0050 train_acc:0.5208\n",
      "epoch: 1 batch:  37 train_loss:5.0036 train_acc:0.5068\n",
      "epoch: 1 batch:  38 train_loss:5.0004 train_acc:0.6579\n",
      "epoch: 1 batch:  39 train_loss:5.0009 train_acc:0.6410\n",
      "epoch: 1 batch:  40 train_loss:5.0010 train_acc:0.6250\n",
      "epoch: 1 batch:  41 train_loss:4.9992 train_acc:0.7622\n",
      "epoch: 1 batch:  42 train_loss:4.9957 train_acc:0.7440\n",
      "epoch: 1 batch:  43 train_loss:4.9945 train_acc:0.7267\n",
      "epoch: 1 batch:  44 train_loss:4.9982 train_acc:0.7102\n",
      "epoch: 1 batch:  45 train_loss:4.9947 train_acc:0.6944\n",
      "epoch: 1 batch:  46 train_loss:4.9936 train_acc:0.6793\n",
      "epoch: 1 batch:  47 train_loss:4.9918 train_acc:0.6649\n",
      "epoch: 1 batch:  48 train_loss:4.9901 train_acc:0.7813\n",
      "epoch: 1 batch:  49 train_loss:4.9877 train_acc:0.7653\n",
      "epoch: 1 batch:  50 train_loss:4.9874 train_acc:0.7500\n",
      "epoch: 1 batch:  51 train_loss:4.9867 train_acc:0.7353\n",
      "epoch: 1 batch:  52 train_loss:4.9854 train_acc:0.7212\n",
      "epoch: 1 batch:  53 train_loss:4.9855 train_acc:0.7075\n",
      "epoch: 1 batch:  54 train_loss:4.9842 train_acc:0.6944\n",
      "epoch: 1 batch:  55 train_loss:4.9847 train_acc:0.6818\n",
      "epoch: 1 batch:  56 train_loss:4.9835 train_acc:0.6696\n",
      "epoch: 1 batch:  57 train_loss:4.9848 train_acc:0.6579\n",
      "epoch: 1 batch:  58 train_loss:4.9840 train_acc:0.7543\n",
      "epoch: 1 batch:  59 train_loss:4.9839 train_acc:0.7415\n",
      "epoch: 1 batch:  60 train_loss:4.9840 train_acc:0.7292\n",
      "epoch: 1 batch:  61 train_loss:4.9849 train_acc:0.7172\n",
      "epoch: 1 batch:  62 train_loss:4.9849 train_acc:0.7056\n",
      "epoch: 1 batch:  63 train_loss:4.9852 train_acc:0.6944\n",
      "epoch: 1 batch:  64 train_loss:4.9857 train_acc:0.6836\n",
      "epoch: 1 batch:  65 train_loss:4.9861 train_acc:0.6731\n",
      "epoch: 1 batch:  66 train_loss:4.9839 train_acc:0.6629\n",
      "epoch: 1 batch:  67 train_loss:4.9824 train_acc:0.6530\n",
      "epoch: 1 batch:  68 train_loss:4.9833 train_acc:0.6434\n",
      "epoch: 1 batch:  69 train_loss:4.9831 train_acc:0.6341\n",
      "epoch: 1 batch:  70 train_loss:4.9826 train_acc:0.6250\n",
      "epoch: 1 batch:  71 train_loss:4.9799 train_acc:0.6162\n",
      "epoch: 1 batch:  72 train_loss:4.9788 train_acc:0.6076\n",
      "epoch: 1 batch:  73 train_loss:4.9789 train_acc:0.5993\n",
      "epoch: 1 batch:  74 train_loss:4.9789 train_acc:0.5912\n",
      "epoch: 1 batch:  75 train_loss:4.9777 train_acc:0.5833\n",
      "epoch: 1 batch:  76 train_loss:4.9759 train_acc:0.5757\n",
      "epoch: 1 batch:  77 train_loss:4.9757 train_acc:0.5682\n",
      "epoch: 1 batch:  78 train_loss:4.9743 train_acc:0.6410\n",
      "epoch: 1 batch:  79 train_loss:4.9735 train_acc:0.6329\n",
      "epoch: 1 batch:  80 train_loss:4.9725 train_acc:0.6250\n",
      "epoch: 1 batch:  81 train_loss:4.9731 train_acc:0.6173\n",
      "epoch: 1 batch:  82 train_loss:4.9724 train_acc:0.6098\n",
      "epoch: 1 batch:  83 train_loss:4.9724 train_acc:0.6024\n",
      "epoch: 1 batch:  84 train_loss:4.9716 train_acc:0.5952\n",
      "epoch: 1 batch:  85 train_loss:4.9709 train_acc:0.5882\n",
      "epoch: 1 batch:  86 train_loss:4.9707 train_acc:0.5814\n",
      "epoch: 1 batch:  87 train_loss:4.9716 train_acc:0.5747\n",
      "epoch: 1 batch:  88 train_loss:4.9696 train_acc:0.5682\n",
      "epoch: 1 batch:  89 train_loss:4.9690 train_acc:0.5618\n",
      "epoch: 1 batch:  90 train_loss:4.9690 train_acc:0.5556\n",
      "epoch: 1 batch:  91 train_loss:4.9684 train_acc:0.5495\n",
      "epoch: 1 batch:  92 train_loss:4.9689 train_acc:0.5435\n",
      "epoch: 1 batch:  93 train_loss:4.9685 train_acc:0.5376\n",
      "epoch: 1 batch:  94 train_loss:4.9681 train_acc:0.5984\n",
      "epoch: 1 batch:  95 train_loss:4.9680 train_acc:0.6579\n",
      "epoch: 1 batch:  96 train_loss:4.9666 train_acc:0.6510\n",
      "epoch: 1 batch:  97 train_loss:4.9652 train_acc:0.6443\n",
      "epoch: 1 batch:  98 train_loss:4.9645 train_acc:0.6378\n",
      "epoch: 1 batch:  99 train_loss:4.9653 train_acc:0.6313\n",
      "epoch: 1 batch: 100 train_loss:4.9645 train_acc:0.6250\n",
      "epoch: 1 batch: 101 train_loss:4.9641 train_acc:0.6807\n",
      "epoch: 1 batch: 102 train_loss:4.9634 train_acc:0.6740\n",
      "epoch: 1 batch: 103 train_loss:4.9629 train_acc:0.6675\n",
      "epoch: 1 batch: 104 train_loss:4.9621 train_acc:0.6611\n",
      "epoch: 1 batch: 105 train_loss:4.9627 train_acc:0.6548\n",
      "epoch: 1 batch: 106 train_loss:4.9626 train_acc:0.6486\n",
      "epoch: 1 batch: 107 train_loss:4.9623 train_acc:0.6425\n",
      "epoch: 1 batch: 108 train_loss:4.9631 train_acc:0.6366\n",
      "epoch: 1 batch: 109 train_loss:4.9637 train_acc:0.6881\n",
      "epoch: 1 batch: 110 train_loss:4.9632 train_acc:0.6818\n",
      "epoch: 1 batch: 111 train_loss:4.9637 train_acc:0.6757\n",
      "epoch: 1 batch: 112 train_loss:4.9638 train_acc:0.6696\n",
      "epoch: 1 batch: 113 train_loss:4.9626 train_acc:0.7190\n",
      "epoch: 1 batch: 114 train_loss:4.9619 train_acc:0.7127\n",
      "epoch: 1 batch: 115 train_loss:4.9626 train_acc:0.7065\n",
      "epoch: 1 batch: 116 train_loss:4.9621 train_acc:0.7004\n",
      "epoch: 1 batch: 117 train_loss:4.9624 train_acc:0.6944\n",
      "epoch: 1 batch: 118 train_loss:4.9623 train_acc:0.6886\n",
      "epoch: 1 batch: 119 train_loss:4.9617 train_acc:0.6828\n",
      "epoch: 1 batch: 120 train_loss:4.9610 train_acc:0.6771\n",
      "epoch: 1 batch: 121 train_loss:4.9604 train_acc:0.6715\n",
      "epoch: 1 batch: 122 train_loss:4.9606 train_acc:0.6660\n",
      "epoch: 1 batch: 123 train_loss:4.9593 train_acc:0.6606\n",
      "epoch: 1 batch: 124 train_loss:4.9595 train_acc:0.6552\n",
      "epoch: 1 batch: 125 train_loss:4.9579 train_acc:0.7000\n",
      "epoch: 1 batch: 126 train_loss:4.9574 train_acc:0.6944\n",
      "epoch: 1 batch: 127 train_loss:4.9569 train_acc:0.7382\n",
      "epoch: 1 batch: 128 train_loss:4.9560 train_acc:0.7324\n",
      "epoch: 1 batch: 129 train_loss:4.9555 train_acc:0.7752\n",
      "epoch: 1 batch: 130 train_loss:4.9560 train_acc:0.8654\n",
      "epoch: 1 batch: 131 train_loss:4.9555 train_acc:0.8588\n",
      "epoch: 1 batch: 132 train_loss:4.9553 train_acc:0.8523\n",
      "epoch: 1 batch: 133 train_loss:4.9551 train_acc:0.8459\n",
      "epoch: 1 batch: 134 train_loss:4.9545 train_acc:0.8396\n",
      "epoch: 1 batch: 135 train_loss:4.9548 train_acc:0.8333\n",
      "epoch: 1 batch: 136 train_loss:4.9543 train_acc:0.8732\n",
      "epoch: 1 batch: 137 train_loss:4.9547 train_acc:0.8668\n",
      "epoch: 1 batch: 138 train_loss:4.9540 train_acc:0.8605\n",
      "epoch: 1 batch: 139 train_loss:4.9544 train_acc:0.8543\n",
      "epoch: 1 batch: 140 train_loss:4.9543 train_acc:0.8482\n",
      "epoch: 1 batch: 141 train_loss:4.9542 train_acc:0.8422\n",
      "epoch: 1 batch: 142 train_loss:4.9539 train_acc:0.8363\n",
      "epoch: 1 batch: 143 train_loss:4.9543 train_acc:0.8304\n",
      "epoch: 1 batch: 144 train_loss:4.9537 train_acc:0.8247\n",
      "epoch: 1 batch: 145 train_loss:4.9525 train_acc:0.8190\n",
      "epoch: 1 batch: 146 train_loss:4.9528 train_acc:0.8134\n",
      "epoch: 1 batch: 147 train_loss:4.9528 train_acc:0.8078\n",
      "epoch: 1 batch: 148 train_loss:4.9521 train_acc:0.8024\n",
      "epoch: 1 batch: 149 train_loss:4.9516 train_acc:0.7970\n",
      "epoch: 1 batch: 150 train_loss:4.9511 train_acc:0.7917\n",
      "epoch: 1 batch: 151 train_loss:4.9508 train_acc:0.7864\n",
      "epoch: 1 batch: 152 train_loss:4.9509 train_acc:0.7812\n",
      "epoch: 1 batch: 153 train_loss:4.9509 train_acc:0.7761\n",
      "epoch: 1 batch: 154 train_loss:4.9511 train_acc:0.7711\n",
      "epoch: 1 batch: 155 train_loss:4.9498 train_acc:0.8065\n",
      "epoch: 1 batch: 156 train_loss:4.9489 train_acc:0.8413\n",
      "epoch: 1 batch: 157 train_loss:4.9491 train_acc:0.8360\n",
      "epoch: 1 batch: 158 train_loss:4.9490 train_acc:0.8307\n",
      "epoch: 1 batch: 159 train_loss:4.9486 train_acc:0.8648\n",
      "epoch: 1 batch: 160 train_loss:4.9476 train_acc:0.8594\n",
      "epoch: 1 batch: 161 train_loss:4.9470 train_acc:0.8540\n",
      "epoch: 1 batch: 162 train_loss:4.9465 train_acc:0.8488\n",
      "epoch: 1 batch: 163 train_loss:4.9463 train_acc:0.8436\n",
      "epoch: 1 batch: 164 train_loss:4.9461 train_acc:0.8384\n",
      "epoch: 1 batch: 165 train_loss:4.9449 train_acc:0.8333\n",
      "epoch: 1 batch: 166 train_loss:4.9443 train_acc:0.8283\n",
      "epoch: 1 batch: 167 train_loss:4.9435 train_acc:0.8608\n",
      "epoch: 1 batch: 168 train_loss:4.9436 train_acc:0.8557\n",
      "epoch: 1 batch: 169 train_loss:4.9437 train_acc:0.8876\n",
      "epoch: 1 batch: 170 train_loss:4.9437 train_acc:0.8824\n",
      "epoch: 1 batch: 171 train_loss:4.9428 train_acc:0.8772\n",
      "epoch: 1 batch: 172 train_loss:4.9423 train_acc:0.8721\n",
      "epoch: 1 batch: 173 train_loss:4.9417 train_acc:0.9032\n",
      "epoch: 1 batch: 174 train_loss:4.9420 train_acc:0.8980\n",
      "epoch: 1 batch: 175 train_loss:4.9413 train_acc:0.9286\n",
      "epoch: 1 batch: 176 train_loss:4.9404 train_acc:0.9588\n",
      "epoch: 1 batch: 177 train_loss:4.9395 train_acc:0.9534\n",
      "epoch: 1 batch: 178 train_loss:4.9390 train_acc:0.9480\n",
      "epoch: 1 batch: 179 train_loss:4.9392 train_acc:0.9427\n",
      "epoch: 1 batch: 180 train_loss:4.9392 train_acc:0.9375\n",
      "epoch: 1 batch: 181 train_loss:4.9386 train_acc:0.9323\n",
      "epoch: 1 batch: 182 train_loss:4.9386 train_acc:0.9615\n",
      "epoch: 1 batch: 183 train_loss:4.9380 train_acc:0.9563\n",
      "epoch: 1 batch: 184 train_loss:4.9379 train_acc:0.9511\n",
      "epoch: 1 batch: 185 train_loss:4.9378 train_acc:0.9459\n",
      "epoch: 1 batch: 186 train_loss:4.9376 train_acc:0.9745\n",
      "epoch: 1 batch: 187 train_loss:4.9380 train_acc:0.9693\n",
      "epoch: 1 batch: 188 train_loss:4.9387 train_acc:0.9641\n",
      "epoch: 1 batch: 189 train_loss:4.9391 train_acc:0.9590\n",
      "epoch: 1 batch: 190 train_loss:4.9390 train_acc:0.9539\n",
      "epoch: 1 batch: 191 train_loss:4.9384 train_acc:0.9490\n",
      "epoch: 1 batch: 192 train_loss:4.9387 train_acc:0.9440\n",
      "epoch: 1 batch: 193 train_loss:4.9380 train_acc:0.9391\n",
      "epoch: 1 batch: 194 train_loss:4.9378 train_acc:0.9665\n",
      "epoch: 1 batch: 195 train_loss:4.9368 train_acc:0.9936\n",
      "epoch: 1 batch: 196 train_loss:4.9366 train_acc:0.9885\n",
      "epoch: 1 batch: 197 train_loss:4.9361 train_acc:1.0152\n",
      "epoch: 1 batch: 198 train_loss:4.9359 train_acc:1.0101\n",
      "epoch: 1 batch: 199 train_loss:4.9366 train_acc:1.0050\n",
      "epoch: 1 batch: 200 train_loss:4.9364 train_acc:1.0000\n",
      "epoch: 1 batch: 201 train_loss:4.9359 train_acc:0.9950\n",
      "epoch: 1 batch: 202 train_loss:4.9350 train_acc:0.9901\n",
      "epoch: 1 batch: 203 train_loss:4.9352 train_acc:1.0160\n",
      "epoch: 1 batch: 204 train_loss:4.9348 train_acc:1.0110\n",
      "epoch: 1 batch: 205 train_loss:4.9343 train_acc:1.0061\n",
      "epoch: 1 batch: 206 train_loss:4.9339 train_acc:1.0316\n",
      "epoch: 1 batch: 207 train_loss:4.9339 train_acc:1.0266\n",
      "epoch: 1 batch: 208 train_loss:4.9339 train_acc:1.0517\n",
      "epoch: 1 batch: 209 train_loss:4.9335 train_acc:1.1065\n",
      "epoch: 1 batch: 210 train_loss:4.9324 train_acc:1.1310\n",
      "epoch: 1 batch: 211 train_loss:4.9319 train_acc:1.1848\n",
      "epoch: 1 batch: 212 train_loss:4.9315 train_acc:1.1792\n",
      "epoch: 1 batch: 213 train_loss:4.9312 train_acc:1.2031\n",
      "epoch: 1 batch: 214 train_loss:4.9314 train_acc:1.1974\n",
      "epoch: 1 batch: 215 train_loss:4.9312 train_acc:1.1919\n",
      "epoch: 1 batch: 216 train_loss:4.9314 train_acc:1.1863\n",
      "epoch: 1 batch: 217 train_loss:4.9306 train_acc:1.2097\n",
      "epoch: 1 batch: 218 train_loss:4.9304 train_acc:1.2328\n",
      "epoch: 1 batch: 219 train_loss:4.9301 train_acc:1.2272\n",
      "epoch: 1 batch: 220 train_loss:4.9299 train_acc:1.2216\n",
      "epoch: 1 batch: 221 train_loss:4.9293 train_acc:1.2161\n",
      "epoch: 1 batch: 222 train_loss:4.9295 train_acc:1.2106\n",
      "epoch: 1 batch: 223 train_loss:4.9294 train_acc:1.2052\n",
      "epoch: 1 batch: 224 train_loss:4.9289 train_acc:1.2277\n",
      "epoch: 1 batch: 225 train_loss:4.9286 train_acc:1.2222\n",
      "epoch: 1 batch: 226 train_loss:4.9285 train_acc:1.2168\n",
      "epoch: 1 batch: 227 train_loss:4.9287 train_acc:1.2115\n",
      "epoch: 1 batch: 228 train_loss:4.9279 train_acc:1.2336\n",
      "epoch: 1 batch: 229 train_loss:4.9274 train_acc:1.2555\n",
      "epoch: 1 batch: 230 train_loss:4.9278 train_acc:1.2500\n",
      "epoch: 1 batch: 231 train_loss:4.9274 train_acc:1.2716\n",
      "epoch: 1 batch: 232 train_loss:4.9267 train_acc:1.2662\n",
      "epoch: 1 batch: 233 train_loss:4.9262 train_acc:1.2607\n",
      "epoch: 1 batch: 234 train_loss:4.9264 train_acc:1.2553\n",
      "epoch: 1 batch: 235 train_loss:4.9266 train_acc:1.2500\n",
      "epoch: 1 batch: 236 train_loss:4.9256 train_acc:1.2712\n",
      "epoch: 1 batch: 237 train_loss:4.9252 train_acc:1.2658\n",
      "epoch: 1 batch: 238 train_loss:4.9249 train_acc:1.2605\n",
      "epoch: 1 batch: 239 train_loss:4.9245 train_acc:1.2552\n",
      "epoch: 1 batch: 240 train_loss:4.9238 train_acc:1.2500\n",
      "epoch: 1 batch: 241 train_loss:4.9235 train_acc:1.2967\n",
      "epoch: 1 batch: 242 train_loss:4.9229 train_acc:1.3430\n",
      "epoch: 1 batch: 243 train_loss:4.9225 train_acc:1.3374\n",
      "epoch: 1 batch: 244 train_loss:4.9225 train_acc:1.3576\n",
      "epoch: 1 batch: 245 train_loss:4.9224 train_acc:1.3520\n",
      "epoch: 1 batch: 246 train_loss:4.9224 train_acc:1.3720\n",
      "epoch: 1 batch: 247 train_loss:4.9225 train_acc:1.3917\n",
      "epoch: 1 batch: 248 train_loss:4.9223 train_acc:1.3861\n",
      "epoch: 1 batch: 249 train_loss:4.9215 train_acc:1.3805\n",
      "epoch: 1 batch: 250 train_loss:4.9219 train_acc:1.3750\n",
      "epoch: 1 batch: 251 train_loss:4.9212 train_acc:1.3695\n",
      "epoch: 1 batch: 252 train_loss:4.9218 train_acc:1.3641\n",
      "epoch: 1 batch: 253 train_loss:4.9212 train_acc:1.3587\n",
      "epoch: 1 batch: 254 train_loss:4.9207 train_acc:1.3533\n",
      "epoch: 1 batch: 255 train_loss:4.9200 train_acc:1.3725\n",
      "epoch: 1 batch: 256 train_loss:4.9194 train_acc:1.3672\n",
      "epoch: 1 batch: 257 train_loss:4.9192 train_acc:1.3862\n",
      "epoch: 1 batch: 258 train_loss:4.9195 train_acc:1.3808\n",
      "epoch: 1 batch: 259 train_loss:4.9196 train_acc:1.3755\n",
      "epoch: 1 batch: 260 train_loss:4.9191 train_acc:1.3942\n",
      "epoch: 1 batch: 261 train_loss:4.9185 train_acc:1.3889\n",
      "epoch: 1 batch: 262 train_loss:4.9187 train_acc:1.3836\n",
      "epoch: 1 batch: 263 train_loss:4.9187 train_acc:1.3783\n",
      "epoch: 1 batch: 264 train_loss:4.9180 train_acc:1.3731\n",
      "epoch: 1 batch: 265 train_loss:4.9173 train_acc:1.3915\n",
      "epoch: 1 batch: 266 train_loss:4.9174 train_acc:1.4098\n",
      "epoch: 1 batch: 267 train_loss:4.9165 train_acc:1.4279\n",
      "epoch: 1 batch: 268 train_loss:4.9162 train_acc:1.4459\n",
      "epoch: 1 batch: 269 train_loss:4.9156 train_acc:1.4405\n",
      "epoch: 1 batch: 270 train_loss:4.9150 train_acc:1.4352\n",
      "epoch: 1 batch: 271 train_loss:4.9146 train_acc:1.4299\n",
      "epoch: 1 batch: 272 train_loss:4.9139 train_acc:1.4476\n",
      "epoch: 1 batch: 273 train_loss:4.9132 train_acc:1.4881\n",
      "epoch: 1 batch: 274 train_loss:4.9130 train_acc:1.4827\n",
      "epoch: 1 batch: 275 train_loss:4.9129 train_acc:1.4773\n",
      "epoch: 1 batch: 276 train_loss:4.9130 train_acc:1.4946\n",
      "epoch: 1 batch: 277 train_loss:4.9134 train_acc:1.4892\n",
      "epoch: 1 batch: 278 train_loss:4.9128 train_acc:1.4838\n",
      "epoch: 1 batch: 279 train_loss:4.9133 train_acc:1.4785\n",
      "epoch: 1 batch: 280 train_loss:4.9129 train_acc:1.4732\n",
      "epoch: 1 batch: 281 train_loss:4.9129 train_acc:1.4902\n",
      "epoch: 1 batch: 282 train_loss:4.9135 train_acc:1.4849\n",
      "epoch: 1 batch: 283 train_loss:4.9131 train_acc:1.4797\n",
      "epoch: 1 batch: 284 train_loss:4.9128 train_acc:1.4745\n",
      "epoch: 1 batch: 285 train_loss:4.9129 train_acc:1.4693\n",
      "epoch: 1 batch: 286 train_loss:4.9127 train_acc:1.4642\n",
      "epoch: 1 batch: 287 train_loss:4.9118 train_acc:1.4808\n",
      "epoch: 1 batch: 288 train_loss:4.9115 train_acc:1.4757\n",
      "epoch: 1 batch: 289 train_loss:4.9116 train_acc:1.4706\n",
      "epoch: 1 batch: 290 train_loss:4.9114 train_acc:1.4655\n",
      "epoch: 1 batch: 291 train_loss:4.9114 train_acc:1.4605\n",
      "epoch: 1 batch: 292 train_loss:4.9107 train_acc:1.4555\n",
      "epoch: 1 batch: 293 train_loss:4.9101 train_acc:1.4505\n",
      "epoch: 1 batch: 294 train_loss:4.9101 train_acc:1.4456\n",
      "epoch: 1 batch: 295 train_loss:4.9097 train_acc:1.4619\n",
      "epoch: 1 batch: 296 train_loss:4.9090 train_acc:1.4569\n",
      "epoch: 1 batch: 297 train_loss:4.9095 train_acc:1.4520\n",
      "epoch: 1 batch: 298 train_loss:4.9095 train_acc:1.4471\n",
      "epoch: 1 batch: 299 train_loss:4.9091 train_acc:1.4423\n",
      "epoch: 1 batch: 300 train_loss:4.9091 train_acc:1.4375\n",
      "epoch: 1 batch: 301 train_loss:4.9093 train_acc:1.4327\n",
      "epoch: 1 batch: 302 train_loss:4.9093 train_acc:1.4280\n",
      "epoch: 1 batch: 303 train_loss:4.9091 train_acc:1.4233\n",
      "epoch: 1 batch: 304 train_loss:4.9086 train_acc:1.4186\n",
      "epoch: 1 batch: 305 train_loss:4.9080 train_acc:1.4344\n",
      "epoch: 1 batch: 306 train_loss:4.9077 train_acc:1.4297\n",
      "epoch: 1 batch: 307 train_loss:4.9077 train_acc:1.4251\n",
      "epoch: 1 batch: 308 train_loss:4.9079 train_acc:1.4205\n",
      "epoch: 1 batch: 309 train_loss:4.9075 train_acc:1.4361\n",
      "epoch: 1 batch: 310 train_loss:4.9069 train_acc:1.4516\n",
      "epoch: 1 batch: 311 train_loss:4.9063 train_acc:1.4871\n",
      "epoch: 1 batch: 312 train_loss:4.9054 train_acc:1.4824\n",
      "epoch: 1 batch: 313 train_loss:4.9050 train_acc:1.4776\n",
      "epoch: 1 batch: 314 train_loss:4.9047 train_acc:1.4729\n",
      "epoch: 1 batch: 315 train_loss:4.9047 train_acc:1.4683\n",
      "epoch: 1 batch: 316 train_loss:4.9044 train_acc:1.5032\n",
      "epoch: 1 batch: 317 train_loss:4.9037 train_acc:1.5181\n",
      "epoch: 1 batch: 318 train_loss:4.9031 train_acc:1.5330\n",
      "epoch: 1 batch: 319 train_loss:4.9031 train_acc:1.5282\n",
      "epoch: 1 batch: 320 train_loss:4.9028 train_acc:1.5234\n",
      "epoch: 1 batch: 321 train_loss:4.9025 train_acc:1.5187\n",
      "epoch: 1 batch: 322 train_loss:4.9026 train_acc:1.5140\n",
      "epoch: 1 batch: 323 train_loss:4.9023 train_acc:1.5093\n",
      "epoch: 1 batch: 324 train_loss:4.9017 train_acc:1.5046\n",
      "epoch: 1 batch: 325 train_loss:4.9014 train_acc:1.5192\n",
      "epoch: 1 batch: 326 train_loss:4.9012 train_acc:1.5146\n",
      "epoch: 1 batch: 327 train_loss:4.9009 train_acc:1.5099\n",
      "epoch: 1 batch: 328 train_loss:4.9010 train_acc:1.5053\n",
      "epoch: 1 batch: 329 train_loss:4.9002 train_acc:1.5388\n",
      "epoch: 1 batch: 330 train_loss:4.8999 train_acc:1.5530\n",
      "epoch: 1 batch: 331 train_loss:4.8999 train_acc:1.5483\n",
      "epoch: 1 batch: 332 train_loss:4.8996 train_acc:1.5437\n",
      "epoch: 1 batch: 333 train_loss:4.8997 train_acc:1.5390\n",
      "epoch: 1 batch: 334 train_loss:4.8992 train_acc:1.5531\n",
      "epoch: 1 batch: 335 train_loss:4.8993 train_acc:1.5485\n",
      "epoch: 1 batch: 336 train_loss:4.8995 train_acc:1.5439\n",
      "epoch: 1 batch: 337 train_loss:4.8992 train_acc:1.5579\n",
      "epoch: 1 batch: 338 train_loss:4.8991 train_acc:1.5533\n",
      "epoch: 1 batch: 339 train_loss:4.8991 train_acc:1.5487\n",
      "epoch: 1 batch: 340 train_loss:4.8993 train_acc:1.5441\n",
      "epoch: 1 batch: 341 train_loss:4.8989 train_acc:1.5946\n",
      "epoch: 1 batch: 342 train_loss:4.8986 train_acc:1.5899\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":    \n",
    "    # BERT\n",
    "#     model_setting = {\n",
    "#         \"model_name\":\"bert\", \n",
    "#         \"config_file_path\":\"bert-base-chinese\", \n",
    "#         \"model_file_path\":\"bert-base-chinese\", \n",
    "#         \"vocab_file_path\":\"bert-base-chinese-vocab.txt\",\n",
    "#         \"num_labels\":149  # 分幾類 \n",
    "#     }    \n",
    "\n",
    "    # ALBERT\n",
    "    model_setting = {\n",
    "        \"model_name\":\"albert\", \n",
    "        \"config_file_path\":\"albert/albert_tiny/config.json\", \n",
    "        \"model_file_path\":\"albert/albert_tiny/pytorch_model.bin\", \n",
    "        \"vocab_file_path\":\"albert/albert_tiny/vocab.txt\",\n",
    "        \"num_labels\":149 # 分幾類\n",
    "    }    \n",
    "\n",
    "    #\n",
    "    model, tokenizer = use_model(**model_setting)\n",
    "    \n",
    "    # setting device    \n",
    "    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "    print(\"using device\",device)\n",
    "    model.to(device)\n",
    "\n",
    "    #    \n",
    "    data_feature = convert_data_to_feature(tokenizer,'Taipei_QA_new.txt')\n",
    "    input_ids = data_feature['input_ids']\n",
    "    input_masks = data_feature['input_masks']\n",
    "    input_segment_ids = data_feature['input_segment_ids']\n",
    "    answer_lables = data_feature['answer_lables']\n",
    "    \n",
    "    #\n",
    "    full_dataset = make_dataset(input_ids = input_ids, input_masks = input_masks, input_segment_ids = input_segment_ids, answer_lables = answer_lables)\n",
    "    train_dataset, test_dataset = split_dataset(full_dataset, 0.9)\n",
    "    train_dataloader = DataLoader(train_dataset,batch_size=16,shuffle=True)\n",
    "    test_dataloader = DataLoader(test_dataset,batch_size=16,shuffle=True)    \n",
    "\n",
    "    # Prepare optimizer and schedule (linear warmup and decay)\n",
    "    no_decay = ['bias', 'LayerNorm.weight']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.0},\n",
    "        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "        ]\n",
    "    optimizer = AdamW(optimizer_grouped_parameters, lr=5e-6, eps=1e-8)\n",
    "    # scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)\n",
    "\n",
    "    model.zero_grad()\n",
    "    for epoch in range(30):\n",
    "        running_loss_val = 0.0\n",
    "        running_acc = 0.0\n",
    "        for batch_index, batch_dict in enumerate(train_dataloader):\n",
    "            model.train()\n",
    "            batch_dict = tuple(t.to(device) for t in batch_dict)\n",
    "            outputs = model(\n",
    "                batch_dict[0],\n",
    "                # attention_mask=batch_dict[1],\n",
    "                labels = batch_dict[3]\n",
    "                )\n",
    "            loss,logits = outputs[:2]\n",
    "            loss.sum().backward()\n",
    "            optimizer.step()\n",
    "            # scheduler.step()  # Update learning rate schedule\n",
    "            model.zero_grad()\n",
    "            \n",
    "            # compute the loss\n",
    "            loss_t = loss.item()\n",
    "            running_loss_val += (loss_t - running_loss_val) / (batch_index + 1)\n",
    "\n",
    "            # compute the accuracy\n",
    "            acc_t = compute_accuracy(logits, batch_dict[3])\n",
    "            running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "\n",
    "            # log\n",
    "            print(\"epoch:%2d batch:%4d train_loss:%2.4f train_acc:%3.4f\"%(epoch+1, batch_index+1, running_loss_val, running_acc))\n",
    "        \n",
    "        running_loss_val = 0.0\n",
    "        running_acc = 0.0\n",
    "        for batch_index, batch_dict in enumerate(test_dataloader):\n",
    "            model.eval()\n",
    "            batch_dict = tuple(t.to(device) for t in batch_dict)\n",
    "            outputs = model(\n",
    "                batch_dict[0],\n",
    "                # attention_mask=batch_dict[1],\n",
    "                labels = batch_dict[3]\n",
    "                )\n",
    "            loss,logits = outputs[:2]\n",
    "            \n",
    "            # compute the loss\n",
    "            loss_t = loss.item()\n",
    "            running_loss_val += (loss_t - running_loss_val) / (batch_index + 1)\n",
    "\n",
    "            # compute the accuracy\n",
    "            acc_t = compute_accuracy(logits, batch_dict[3])\n",
    "            running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "\n",
    "            # log\n",
    "            print(\"epoch:%2d batch:%4d test_loss:%2.4f test_acc:%3.4f\"%(epoch+1, batch_index+1, running_loss_val, running_acc))\n",
    "    \n",
    "    model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n",
    "    model_to_save.save_pretrained('trained_model')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
